{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import re\n",
    "import csv\n",
    "import urllib.request\n",
    "import urllib.parse\n",
    "import urllib.robotparser\n",
    "from bs4 import BeautifulSoup\n",
    "import lxml.html\n",
    "from lxml.cssselect import CSSSelector\n",
    "import os\n",
    "import pickle\n",
    "import zlib\n",
    "import shutil\n",
    "from datetime import datetime, timedelta\n",
    "from pymongo import MongoClient\n",
    "from bson.binary import Binary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Throttle:\n",
    "    \"\"\"\n",
    "    Add a delay between downloads to the same domain\n",
    "    \"\"\"\n",
    "    def __init__(self,delay):\n",
    "        # amount of delay between downloads for each domain\n",
    "        self.delay = delay\n",
    "        # timestamp of when a domain was last accessed\n",
    "        self.domains = {}\n",
    "        \n",
    "    def wait(self,url):\n",
    "        domain = urllib.parse.urlparse(url).netloc\n",
    "        last_accessed = self.domains.get(domain)\n",
    "        if self.delay > 0 and last_accessed is not None:\n",
    "            sleep_secs = self.delay - (datetime.now() - last_accessed).seconds\n",
    "            if sleep_secs > 0:\n",
    "                # domain has been accessed recently so need to sleep\n",
    "                time.sleep(sleep_secs)\n",
    "        # update the last accessed time\n",
    "        self.domains[domain] = datetime.now()\n",
    "class Downloader:\n",
    "    def __init__(self, delay=5, user_agent='wswp', proxies=None, num_retries=1, cache=None):\n",
    "        self.throttle = Throttle(delay)\n",
    "        self.user_agent = user_agent\n",
    "        self.proxies = proxies\n",
    "        self.num_retries = num_retries\n",
    "        self.cache = cache\n",
    "    def __call__(self, url):\n",
    "        result = None\n",
    "        if self.cache:\n",
    "            try:\n",
    "                result = self.cache[url]\n",
    "            except KeyError:\n",
    "                # url is not available in cache\n",
    "                pass\n",
    "            else:\n",
    "                if self.num_retries > 0 and 500 <= result['code'] < 600:\n",
    "                    #server error so ignore result from cache and re-download\n",
    "                    result = None\n",
    "        if result is None:\n",
    "            # result was not loaded from cache so still need to download\n",
    "            self.throttle.wait(url)\n",
    "            proxy = random.choice(self.proxies) if self.proxies else None\n",
    "            headers = {'User-agent':self.user_agent}\n",
    "            result = self.download(url, headers, proxy, self.num_retries)\n",
    "            if result is None:\n",
    "                return None\n",
    "            if self.cache:\n",
    "                # save result to cache\n",
    "                self.cache[url] = result\n",
    "        return result['html']\n",
    "    def download(self, url, headers, proxy, num_retries, data=None):\n",
    "        print('Downloading:',url)\n",
    "        request = urllib.request.Request(url,headers=headers)\n",
    "        opener = urllib.request.build_opener()\n",
    "        if proxy:\n",
    "            proxy_params = {urllib.parse.urlparse(url).scheme: proxy}\n",
    "            opener.add_handler(urllib.request.ProxyHandler(proxy_params))\n",
    "        try:\n",
    "            response = opener.open(request)\n",
    "        except urllib.request.URLError as e:\n",
    "            print('Download error:',e.reason)\n",
    "            response = None\n",
    "            if num_retries>0 and hasattr(e,'code') and 500<=e.code<600:\n",
    "                return self.download(url,headers,proxy,num_retries-1)\n",
    "        if response:\n",
    "            return {'html':response.read().decode('utf-8'),'code':response.code}\n",
    "        else:\n",
    "            return None\n",
    "class DiskCache:\n",
    "    def __init__(self, cache_dir='cache', compress=False, expires=timedelta(days=30)):\n",
    "        self.cache_dir = cache_dir\n",
    "        self.compress = compress\n",
    "        self.expires = expires\n",
    "    def url_to_path(self, url):\n",
    "        '''\n",
    "        Create file system path for this URL\n",
    "        '''\n",
    "        components = urllib.parse.urlsplit(url)\n",
    "        # append index.html to empth paths\n",
    "        path = components.path\n",
    "        if not path:\n",
    "            path = '/index'\n",
    "        elif path.endswith('/'):\n",
    "            path += 'index'\n",
    "        path += '.html'\n",
    "        filename = components.netloc + path + components.query\n",
    "        # replace invalid characters\n",
    "        filename = re.sub('[^/0-9a-zA-Z\\-.,;_ ]', '_', filename)\n",
    "        # restrict maximum number of characters\n",
    "        filename = '/'.join(segment[:255] for segment in filename.split('/'))\n",
    "        return os.path.join(self.cache_dir, filename)\n",
    "    def __getitem__(self, url):\n",
    "        \"\"\"\n",
    "        Load data from disk for this URL\n",
    "        \"\"\"\n",
    "        path = self.url_to_path(url)\n",
    "        if os.path.exists(path):\n",
    "            with open(path, 'rb') as fp:\n",
    "                data = fp.read()\n",
    "                if self.compress:\n",
    "                    data = zlib.decompress(data)\n",
    "                result, timestamp = pickle.loads(data)\n",
    "                if self.has_expired(timestamp):\n",
    "                    raise KeyError(url + ' has expired')\n",
    "                return result\n",
    "        else:\n",
    "            # url has not yet been cached\n",
    "            raise KeyError(url + ' does not exist')\n",
    "    def __setitem__(self, url, result):\n",
    "        \"\"\"\n",
    "        Save data to disk for this URL\n",
    "        \"\"\"\n",
    "        path = self.url_to_path(url)\n",
    "        folder = os.path.dirname(path)\n",
    "        if not os.path.exists(folder):\n",
    "            os.makedirs(folder)\n",
    "        data = pickle.dumps((result, datetime.utcnow()))\n",
    "        if self.compress:\n",
    "            data = zlib.compress(data)\n",
    "        with open(path, 'wb') as fp:\n",
    "            fp.write(data)\n",
    "    def __delitem__(self, url):\n",
    "        \"\"\"\n",
    "        Remove the value at this key and any empty parent sub-directories\n",
    "        \"\"\"\n",
    "        path = self._key_path(url)\n",
    "        try:\n",
    "            os.remove(path)\n",
    "            os.removedirs(os.path.dirname(path))\n",
    "        except OSError:\n",
    "            pass\n",
    "    def has_expired(self, timestamp):\n",
    "        \"\"\"Return whether this timestamp has expired\n",
    "        \"\"\"\n",
    "        return datetime.utcnow() > (timestamp + self.expires)\n",
    "    def clear(self):\n",
    "        \"\"\"Remove all the cached values\n",
    "        \"\"\"\n",
    "        if os.path.exists(self.cache_dir):\n",
    "            shutil.rmtree(self.cache_dir)\n",
    "            \n",
    "def link_crawler(send_url, rps=True, link_regex=None, delay=1, max_depth=2, scrape_callback=None, cache=None):\n",
    "    if rps:\n",
    "        rp = urllib.robotparser.RobotFileParser()\n",
    "        rp.set_url(urllib.parse.urljoin(send_url,'/robots.txt'))\n",
    "        rp.read()\n",
    "    user_agent = \"Baiduspider\"\n",
    "    crawl_queue = [send_url]\n",
    "    seen = {send_url:0}\n",
    "    D = Downloader(delay=delay, user_agent=user_agent, proxies=None, num_retries=2, cache=cache)\n",
    "    while crawl_queue:\n",
    "        url = crawl_queue.pop()\n",
    "        depth = seen[url]\n",
    "        if depth != max_depth:\n",
    "            # chec url passes robots.txt restrictions\n",
    "            if rps is False or rp.can_fetch(user_agent,url):\n",
    "                html = D(url)\n",
    "                if html == None:\n",
    "                    continue;\n",
    "                links = []\n",
    "                if scrape_callback:\n",
    "                    links.extend(scrape_callback(url, html) or [])\n",
    "                if link_regex:\n",
    "                    # filter for links matching our regular expression\n",
    "                    links.extend(link for link in get_links(html) if re.match(link_regex, link))\n",
    "                    # print('肯定会执行')\n",
    "                for link in links:\n",
    "                    # from absolute link\n",
    "                    link = urllib.parse.urljoin(send_url,link)\n",
    "                    # check if have alreday seen this link\n",
    "                    if link not in seen:\n",
    "                        seen[link] = depth + 1\n",
    "                        crawl_queue.append(link)\n",
    "            else:\n",
    "                print('Blocked by robots.txt:',url)\n",
    "def get_links(html):\n",
    "    \"\"\"\n",
    "    Return a list of links from html\n",
    "    \"\"\"\n",
    "    # a regular expression to extract all links from the webpage\n",
    "    webpage_regex = re.compile('<a[^>]+href=[\"\\'](.*?)[\"\\']',re.IGNORECASE)\n",
    "    # list of all links from the webpaeg\n",
    "    return webpage_regex.findall(html)\n",
    "FIELDS = ('area', 'population', 'iso', 'country', 'capital', 'continent', 'tld', 'currency_code', 'currency_name', 'phone', 'postal_code_format', 'postal_code_regex', 'languages', 'neighbours')\n",
    "def scrape_callback(url, html):\n",
    "    if re.search('/view/', url):\n",
    "        tree = lxml.html.fromstring(html)\n",
    "        row = [tree.cssselect('table > tr#places_%s__row > td.w2p_fw' % field)[0].text_content() for field in FIELDS]\n",
    "        print(url, row)\n",
    "class AlexaCallback:\n",
    "    def __init__(self, max_urls=1000):\n",
    "        self.max_urls = max_urls\n",
    "        self.seed_url = 'http://s3.amazonaws.com/alexa-static/top-1m.csv.zip'\n",
    "    def __call__(self, url, html):\n",
    "        if url == self.seed_url:\n",
    "            urls = []\n",
    "            with open('top-1m.csv', 'r', encoding='utf-8') as file:\n",
    "                for _, website in csv.reader(file):\n",
    "                    urls.append('http://' + website)\n",
    "                    if len(urls) == 1000:\n",
    "                        break\n",
    "            return urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: https://bitbucket.org/wswp/code\n",
      "2.98 seconds\n"
     ]
    }
   ],
   "source": [
    "cache = DiskCache('code')\n",
    "scrape_callback = AlexaCallback()\n",
    "start = time.time() # record start time of scrape\n",
    "link_crawler('https://bitbucket.org/wswp/code', rps=False, link_regex='src', max_depth=-1, scrape_callback=scrape_callback, cache=cache)    \n",
    "end = time.time()\n",
    "print('%.2f seconds' % (end-start)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import threading\n",
    "\n",
    "SLEEP_TIME = 1\n",
    "\n",
    "def threaded_crawler(send_url, delay=1, cache=None, max_threads=10):\n",
    "    # the queue of URL's that still need to be crawled\n",
    "    crawl_queue = send_url\n",
    "    D = Downloader(cache=cache, delay=delay)\n",
    "    threads = []\n",
    "    while threads or crawl_queue:\n",
    "        # the crawl is still active\n",
    "        for thread in threads:\n",
    "            if not thread.is_alive():\n",
    "                # remove the stopped threads\n",
    "                threads.remove(thread)\n",
    "        def process_queue():\n",
    "            while True:\n",
    "                try:\n",
    "                    url = crawl_queue.pop()\n",
    "                except IndexError:\n",
    "                    # crawl queue is empty\n",
    "                    break\n",
    "                else:\n",
    "                    html = D(url)\n",
    "        while len(threads) < max_threads and crawl_queue:\n",
    "            # can start some more threads\n",
    "            thread = threading.Thread(target=process_queue)\n",
    "            # set daemon so main thread can exit when receives ctrl-c\n",
    "            thread.setDaemon(True)\n",
    "            thread.start()\n",
    "            threads.append(thread)\n",
    "        # all threads have been processed\n",
    "        # sleep temporarily so CPU can focus execution elsewhere\n",
    "        time.sleep(SLEEP_TIME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "urls = []\n",
    "with open('top-1m.csv', 'r', encoding='utf-8') as file:\n",
    "    for _, website in csv.reader(file):\n",
    "        urls.append('http://' + website)\n",
    "        if len(urls) == 1000:\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://zomato.com\n",
      "Downloading: http://suara.com\n",
      "Downloading: http://worldstarhiphop.com\n",
      "Downloading: http://ozon.ru\n",
      "Downloading:Downloading: http://tencent.com\n",
      " http://pulzo.com\n",
      "Downloading: http://google.iq\n",
      "Downloading: http://arynews.tv\n",
      "Downloading: http://mydrivers.com\n",
      "Downloading: http://francetvinfo.fr\n",
      "Downloading: http://lun.com\n",
      "Download error: Forbidden\n",
      "Downloading: http://dw.com\n",
      "Downloading: http://dailypost.ng\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-14:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 67, in download\n",
      "    return {'html':response.read().decode('utf-8'),'code':response.code}\n",
      "UnicodeDecodeError: 'utf-8' codec can't decode byte 0xbf in position 255: invalid start byte\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://irs.gov\n",
      "Downloading: http://getbootstrap.com\n",
      "Downloading: http://canada.ca\n",
      "Downloading: http://admaimai.com\n",
      "Downloading: http://provincial.com\n",
      "Downloading: http://marriott.com\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-15:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 67, in download\n",
      "    return {'html':response.read().decode('utf-8'),'code':response.code}\n",
      "UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb9 in position 150: invalid start byte\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://ebates.com\n",
      "Downloading: http://usaa.com\n",
      "Downloading: http://viber.com\n",
      "Downloading: http://dagospia.com\n",
      "Downloading: http://intentmedia.net\n",
      "Downloading: http://cdstm.cn\n",
      "Downloading: http://jeuxvideo.com\n",
      "Download error: [Errno 11001] getaddrinfo failed\n",
      "Downloading: http://dhgate.com\n",
      "Downloading: http://idnes.cz\n",
      "Downloading: http://google.com.my\n",
      "Downloading: http://okcupid.com\n",
      "Downloading: http://amazon.com.mx\n",
      "Downloading: http://bamilo.com\n",
      "Downloading: http://vesti.ru\n",
      "Downloading: http://creditkarma.com\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-7:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 67, in download\n",
      "    return {'html':response.read().decode('utf-8'),'code':response.code}\n",
      "UnicodeDecodeError: 'utf-8' codec can't decode byte 0x96 in position 393: invalid start byte\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://flaticon.com\n",
      "Downloading: http://google.hr\n",
      "Download error: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:833)\n",
      "Downloading: http://docusign.net\n",
      "Downloading: http://popmyads.com\n",
      "Downloading: http://wondershare.com\n",
      "Downloading: http://google.lt\n",
      "Downloading: http://chron.com\n",
      "Downloading: http://nikkei.com\n",
      "Downloading: http://donga.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://winmoreprize.com\n",
      "Downloading: http://ytimg.com\n",
      "Download error: [Errno 11001] getaddrinfo failed\n",
      "Downloading: http://paytmmall.com\n",
      "Downloading: http://t.me\n",
      "Downloading: http://w3school.com.cn\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://teamviewer.com\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-11:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 67, in download\n",
      "    return {'html':response.read().decode('utf-8'),'code':response.code}\n",
      "UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd4 in position 503: invalid continuation byte\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://tim.it\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-6:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 60, in download\n",
      "    response = opener.open(request)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 532, in open\n",
      "    response = meth(req, response)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 642, in http_response\n",
      "    'http', request, response, code, msg, hdrs)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 564, in error\n",
      "    result = self._call_chain(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 504, in _call_chain\n",
      "    result = func(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 756, in http_error_302\n",
      "    return self.parent.open(new, timeout=req.timeout)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 532, in open\n",
      "    response = meth(req, response)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 642, in http_response\n",
      "    'http', request, response, code, msg, hdrs)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 564, in error\n",
      "    result = self._call_chain(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 504, in _call_chain\n",
      "    result = func(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 756, in http_error_302\n",
      "    return self.parent.open(new, timeout=req.timeout)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 526, in open\n",
      "    response = self._open(req, data)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 544, in _open\n",
      "    '_open', req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 504, in _call_chain\n",
      "    result = func(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1361, in https_open\n",
      "    context=self._context, check_hostname=self._check_hostname)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1321, in do_open\n",
      "    r = h.getresponse()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 1331, in getresponse\n",
      "    response.begin()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 297, in begin\n",
      "    version, status, reason = self._read_status()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 258, in _read_status\n",
      "    line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\socket.py\", line 586, in readinto\n",
      "    return self._sock.recv_into(b)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\ssl.py\", line 1009, in recv_into\n",
      "    return self.read(nbytes, buffer)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\ssl.py\", line 871, in read\n",
      "    return self._sslobj.read(len, buffer)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\ssl.py\", line 631, in read\n",
      "    v = self._sslobj.read(len, buffer)\n",
      "TimeoutError: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://gap.tmall.com\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-19:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 60, in download\n",
      "    response = opener.open(request)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 526, in open\n",
      "    response = self._open(req, data)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 544, in _open\n",
      "    '_open', req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 504, in _call_chain\n",
      "    result = func(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1346, in http_open\n",
      "    return self.do_open(http.client.HTTPConnection, req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1321, in do_open\n",
      "    r = h.getresponse()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 1331, in getresponse\n",
      "    response.begin()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 297, in begin\n",
      "    version, status, reason = self._read_status()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 258, in _read_status\n",
      "    line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\socket.py\", line 586, in readinto\n",
      "    return self._sock.recv_into(b)\n",
      "ConnectionResetError: [WinError 10054] 远程主机强迫关闭了一个现有的连接。\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://medianewpage.com\n",
      "Downloading: http://sapo.pt\n",
      "Download error: The HTTP server returned a redirect error that would lead to an infinite loop.\n",
      "The last 30x error message was:\n",
      "\n",
      "Downloading: http://gotomeeting.com\n",
      "Download error: [WinError 10061] 由于目标计算机积极拒绝，无法连接。\n",
      "Downloading: http://qualtrics.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://nmisr.com\n",
      "Downloading: http://askubuntu.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://nexon.com\n",
      "Downloading: http://biblegateway.com\n",
      "Downloading: http://istockphoto.com\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-20:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 60, in download\n",
      "    response = opener.open(request)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 526, in open\n",
      "    response = self._open(req, data)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 544, in _open\n",
      "    '_open', req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 504, in _call_chain\n",
      "    result = func(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1346, in http_open\n",
      "    return self.do_open(http.client.HTTPConnection, req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1321, in do_open\n",
      "    r = h.getresponse()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 1331, in getresponse\n",
      "    response.begin()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 297, in begin\n",
      "    version, status, reason = self._read_status()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 258, in _read_status\n",
      "    line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\socket.py\", line 586, in readinto\n",
      "    return self._sock.recv_into(b)\n",
      "ConnectionResetError: [WinError 10054] 远程主机强迫关闭了一个现有的连接。\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://metropoles.com\n",
      "Downloading: http://fanfiction.net\n",
      "Downloading: http://thesun.co.uk\n",
      "Downloading: http://cleartax.in\n",
      "Downloading: http://readthedocs.io\n",
      "Downloading: http://redwap.me\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://axisbank.co.in\n",
      "Downloading: http://gazetaexpress.com\n",
      "Download error: [Errno 11001] getaddrinfo failed\n",
      "Downloading: http://jumia.com.ng\n",
      "Downloading: http://reacherinst.com\n",
      "Download error: Forbidden\n",
      "Downloading: http://theatlantic.com\n",
      "Downloading: http://tandfonline.com\n",
      "Downloading: http://ea.com\n",
      "Downloading: http://vporn.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://sportbible.com\n",
      "Downloading: http://buzzfeednews.com\n",
      "Downloading: http://yt-adblocker.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://qj.com.cn\n",
      "Downloading: http://itmedia.co.jp\n",
      "Downloading: http://tabelog.com\n",
      "Downloading: http://nbcnews.com\n",
      "Downloading: http://rumble.com\n",
      "Downloading: http://stanford.edu\n",
      "Downloading: http://mawdoo3.com\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-21:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 67, in download\n",
      "    return {'html':response.read().decode('utf-8'),'code':response.code}\n",
      "UnicodeDecodeError: 'utf-8' codec can't decode byte 0x94 in position 754: invalid start byte\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://letour.fr\n",
      "Downloading: http://duolingo.com\n",
      "Downloading: http://best2018games.com\n",
      "Downloading: http://mobile.de\n",
      "Downloading: http://ecosia.org\n",
      "Downloading: http://howtogeek.com\n",
      "Download error: Forbidden\n",
      "Downloading: http://coinbase.com\n",
      "Downloading: http://thehill.com\n",
      "Downloading: http://news18.com\n",
      "Downloading: http://gamib.com\n",
      "Downloading: http://jqw.com\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-12:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 67, in download\n",
      "    return {'html':response.read().decode('utf-8'),'code':response.code}\n",
      "UnicodeDecodeError: 'utf-8' codec can't decode byte 0xbd in position 83: invalid start byte\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://exhentai.org\n",
      "Downloading: http://eporner.com\n",
      "Downloading: http://strava.com\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-23:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 67, in download\n",
      "    return {'html':response.read().decode('utf-8'),'code':response.code}\n",
      "UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8b in position 1: invalid start byte\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://nasa.gov\n",
      "Downloading: http://prnt.sc\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://gazeta.ru\n",
      "Downloading: http://freejobalert.com\n",
      "Downloading: http://ryanair.com\n",
      "Downloading: http://addroplet.com\n",
      "Downloading: http://digitaltrends.com\n",
      "Downloading: http://wildberries.ru\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-13:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 67, in download\n",
      "    return {'html':response.read().decode('utf-8'),'code':response.code}\n",
      "UnicodeDecodeError: 'utf-8' codec can't decode byte 0xc3 in position 3044: invalid continuation byte\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://aa.com\n",
      "Downloading: http://badoo.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://tdscpc.gov.in\n",
      "Downloading: http://arxiv.org\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-17:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 67, in download\n",
      "    return {'html':response.read().decode('utf-8'),'code':response.code}\n",
      "UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8b in position 1: invalid start byte\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://rutube.ru\n",
      "Downloading: http://snapdeal.com\n",
      "Download error: Moved Temporarily\n",
      "Downloading: http://skysports.com\n",
      "Downloading: http://rajasthan.gov.in\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://turkiye.gov.tr\n",
      "Download error: Forbidden\n",
      "Downloading: http://pornpics.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://zing.vn\n",
      "Downloading: http://macys.com\n",
      "Downloading: http://nhk.or.jp\n",
      "Download error: [Errno 11001] getaddrinfo failed\n",
      "Downloading: http://zcool.com.cn\n",
      "Downloading: http://firefoxchina.cn\n",
      "Downloading: http://tomshardware.com\n",
      "Downloading: http://reclameaqui.com.br\n",
      "Downloading: http://infobae.com\n",
      "Downloading: http://issuu.com\n",
      "Downloading: http://ukr.net\n",
      "Downloading: http://xtube.com\n",
      "Downloading: http://ccm.net\n",
      "Downloading: http://ivi.ru\n",
      "Downloading: http://trulia.com\n",
      "Downloading: http://united.com\n",
      "Downloading: http://pcgamer.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://jb51.net\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-25:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 67, in download\n",
      "    return {'html':response.read().decode('utf-8'),'code':response.code}\n",
      "UnicodeDecodeError: 'utf-8' codec can't decode byte 0xbd in position 303: invalid start byte\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://khanacademy.org\n",
      "Downloading: http://lefigaro.fr\n",
      "Downloading: http://google.com.gt\n",
      "Downloading: http://jamnews.com\n",
      "Downloading: http://11st.co.kr\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-28:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 67, in download\n",
      "    return {'html':response.read().decode('utf-8'),'code':response.code}\n",
      "UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb4 in position 225: invalid start byte\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://prezi.com\n",
      "Downloading: http://visualstudio.com\n",
      "Downloading: http://meetup.com\n",
      "Downloading: http://lowes.com\n",
      "Downloading: http://umblr.com\n",
      "Downloading: http://chip.de\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-10:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 60, in download\n",
      "    response = opener.open(request)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 532, in open\n",
      "    response = meth(req, response)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 642, in http_response\n",
      "    'http', request, response, code, msg, hdrs)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 564, in error\n",
      "    result = self._call_chain(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 504, in _call_chain\n",
      "    result = func(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 756, in http_error_302\n",
      "    return self.parent.open(new, timeout=req.timeout)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 526, in open\n",
      "    response = self._open(req, data)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 544, in _open\n",
      "    '_open', req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 504, in _call_chain\n",
      "    result = func(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1361, in https_open\n",
      "    context=self._context, check_hostname=self._check_hostname)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1321, in do_open\n",
      "    r = h.getresponse()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 1331, in getresponse\n",
      "    response.begin()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 297, in begin\n",
      "    version, status, reason = self._read_status()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 258, in _read_status\n",
      "    line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\socket.py\", line 586, in readinto\n",
      "    return self._sock.recv_into(b)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\ssl.py\", line 1009, in recv_into\n",
      "    return self.read(nbytes, buffer)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\ssl.py\", line 871, in read\n",
      "    return self._sslobj.read(len, buffer)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\ssl.py\", line 631, in read\n",
      "    v = self._sslobj.read(len, buffer)\n",
      "TimeoutError: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://asahi.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://go2affise.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://youjizz.com\n",
      "Downloading: http://auction.co.kr\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-29:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 67, in download\n",
      "    return {'html':response.read().decode('utf-8'),'code':response.code}\n",
      "UnicodeDecodeError: 'utf-8' codec can't decode byte 0xbf in position 188: invalid start byte\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://dmv.org\n",
      "Downloading: http://tilestwra.com\n",
      "Downloading: http://hollywoodreporter.com\n",
      "Downloading: http://wetter.com\n",
      "Downloading: http://td.com\n",
      "Downloading: http://farfetch.com\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-27:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 60, in download\n",
      "    response = opener.open(request)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 532, in open\n",
      "    response = meth(req, response)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 642, in http_response\n",
      "    'http', request, response, code, msg, hdrs)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 564, in error\n",
      "    result = self._call_chain(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 504, in _call_chain\n",
      "    result = func(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 756, in http_error_302\n",
      "    return self.parent.open(new, timeout=req.timeout)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 532, in open\n",
      "    response = meth(req, response)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 642, in http_response\n",
      "    'http', request, response, code, msg, hdrs)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 564, in error\n",
      "    result = self._call_chain(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 504, in _call_chain\n",
      "    result = func(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 756, in http_error_302\n",
      "    return self.parent.open(new, timeout=req.timeout)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 526, in open\n",
      "    response = self._open(req, data)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 544, in _open\n",
      "    '_open', req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 504, in _call_chain\n",
      "    result = func(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1361, in https_open\n",
      "    context=self._context, check_hostname=self._check_hostname)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1321, in do_open\n",
      "    r = h.getresponse()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 1331, in getresponse\n",
      "    response.begin()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 297, in begin\n",
      "    version, status, reason = self._read_status()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 258, in _read_status\n",
      "    line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\socket.py\", line 586, in readinto\n",
      "    return self._sock.recv_into(b)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\ssl.py\", line 1009, in recv_into\n",
      "    return self.read(nbytes, buffer)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\ssl.py\", line 871, in read\n",
      "    return self._sslobj.read(len, buffer)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\ssl.py\", line 631, in read\n",
      "    v = self._sslobj.read(len, buffer)\n",
      "TimeoutError: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://elzmannews.com\n",
      "Downloading: http://rapidgator.net\n",
      "Downloading: http://rarbg.is\n",
      "Downloading: http://scol.com.cn\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://ruliweb.com\n",
      "Downloading: http://westernjournal.com\n",
      "Downloading: http://hatena.ne.jp\n",
      "Downloading: http://motorsport.com\n",
      "Downloading: http://prom.ua\n",
      "Downloading: http://linkshrink.net\n",
      "Downloading: http://cnbeta.com\n",
      "Downloading: http://mynet.com\n",
      "Downloading: http://hepsiburada.com\n",
      "Downloading: http://motherless.com\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-24:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 60, in download\n",
      "    response = opener.open(request)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 526, in open\n",
      "    response = self._open(req, data)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 544, in _open\n",
      "    '_open', req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 504, in _call_chain\n",
      "    result = func(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1346, in http_open\n",
      "    return self.do_open(http.client.HTTPConnection, req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1321, in do_open\n",
      "    r = h.getresponse()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 1331, in getresponse\n",
      "    response.begin()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 297, in begin\n",
      "    version, status, reason = self._read_status()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 258, in _read_status\n",
      "    line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\socket.py\", line 586, in readinto\n",
      "    return self._sock.recv_into(b)\n",
      "ConnectionResetError: [WinError 10054] 远程主机强迫关闭了一个现有的连接。\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://alodokter.com\n",
      "Downloading: http://v2ex.com\n",
      "Download error: Not allowed.\n",
      "Downloading: http://chess.com\n",
      "Downloading: http://gotporn.com\n",
      "Downloading: http://academia.edu\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://apkpure.com\n",
      "Downloading: http://huawei.com\n",
      "Downloading: http://lapatilla.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://yenisafak.com\n",
      "Downloading: http://4pda.ru\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-22:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 60, in download\n",
      "    response = opener.open(request)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 526, in open\n",
      "    response = self._open(req, data)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 544, in _open\n",
      "    '_open', req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 504, in _call_chain\n",
      "    result = func(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1346, in http_open\n",
      "    return self.do_open(http.client.HTTPConnection, req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1321, in do_open\n",
      "    r = h.getresponse()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 1331, in getresponse\n",
      "    response.begin()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 297, in begin\n",
      "    version, status, reason = self._read_status()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 258, in _read_status\n",
      "    line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\socket.py\", line 586, in readinto\n",
      "    return self._sock.recv_into(b)\n",
      "ConnectionResetError: [WinError 10054] 远程主机强迫关闭了一个现有的连接。\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://gmanetwork.com\n",
      "Downloading: http://nianhuo.tmall.com\n",
      "Downloading: http://urdupoint.com\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-33:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 67, in download\n",
      "    return {'html':response.read().decode('utf-8'),'code':response.code}\n",
      "UnicodeDecodeError: 'utf-8' codec can't decode byte 0xcf in position 862: invalid continuation byte\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://zhaopin.com\n",
      "Downloading: http://labanquepostale.fr\n",
      "Downloading: http://oschina.net\n",
      "Downloading: http://codepen.io\n",
      "Downloading: http://bankmellat.ir\n",
      "Downloading: http://n11.com\n",
      "Downloading: http://mileroticos.com\n",
      "Downloading: http://360doc.com\n",
      "Downloading: http://subito.it\n",
      "Downloading: http://dailycaller.com\n",
      "Downloading: http://bhphotovideo.com\n",
      "Downloading: http://nature.com\n",
      "Downloading: http://dictionary.com\n",
      "Downloading: http://segmentfault.com\n",
      "Downloading: http://knowyourmeme.com\n",
      "Downloading: http://chouftv.ma\n",
      "Downloading: http://in.gr\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://cam4.com\n",
      "Downloading: http://realtor.com\n",
      "Downloading: http://ninisite.com\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-31:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 60, in download\n",
      "    response = opener.open(request)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 526, in open\n",
      "    response = self._open(req, data)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 544, in _open\n",
      "    '_open', req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 504, in _call_chain\n",
      "    result = func(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1346, in http_open\n",
      "    return self.do_open(http.client.HTTPConnection, req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1321, in do_open\n",
      "    r = h.getresponse()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 1331, in getresponse\n",
      "    response.begin()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 297, in begin\n",
      "    version, status, reason = self._read_status()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 258, in _read_status\n",
      "    line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\socket.py\", line 586, in readinto\n",
      "    return self._sock.recv_into(b)\n",
      "ConnectionResetError: [WinError 10054] 远程主机强迫关闭了一个现有的连接。\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://usnews.com\n",
      "Downloading: http://subject.tmall.com\n",
      "Downloading: http://cimaclub.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://food.tmall.com\n",
      "Downloading: http://nvzhuang.tmall.com\n",
      "Download error: Forbidden\n",
      "Downloading: http://pchome.com.tw\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-16:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 60, in download\n",
      "    response = opener.open(request)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 526, in open\n",
      "    response = self._open(req, data)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 544, in _open\n",
      "    '_open', req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 504, in _call_chain\n",
      "    result = func(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1346, in http_open\n",
      "    return self.do_open(http.client.HTTPConnection, req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1321, in do_open\n",
      "    r = h.getresponse()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 1331, in getresponse\n",
      "    response.begin()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 297, in begin\n",
      "    version, status, reason = self._read_status()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 258, in _read_status\n",
      "    line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\socket.py\", line 586, in readinto\n",
      "    return self._sock.recv_into(b)\n",
      "ConnectionResetError: [WinError 10054] 远程主机强迫关闭了一个现有的连接。\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://y8.com\n",
      "Downloading: http://caixa.gov.br\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://cdiscount.com\n",
      "Downloading: http://dikaiologitika.gr\n",
      "Download error: The HTTP server returned a redirect error that would lead to an infinite loop.\n",
      "The last 30x error message was:\n",
      "Found\n",
      "Downloading: http://uploaded.net\n",
      "Downloading: http://getpocket.com\n",
      "Downloading: http://dribbble.com\n",
      "Downloading: http://vidio.com\n",
      "Downloading: http://4shared.com\n",
      "Downloading: http://pinterest.co.uk\n",
      "Downloading: http://trustpilot.com\n",
      "Downloading: http://americanas.com.br\n",
      "Download error: The HTTP server returned a redirect error that would lead to an infinite loop.\n",
      "The last 30x error message was:\n",
      "Found\n",
      "Downloading: http://abola.pt\n",
      "Downloading: http://marketwatch.com\n",
      "Downloading: http://lenovo.com\n",
      "Downloading: http://tomsguide.com\n",
      "Download error: Forbidden\n",
      "Downloading: http://google.tn\n",
      "Downloading: http://cnzz.com\n",
      "Downloading: http://verizonwireless.com\n",
      "Downloading: http://elbalad.news\n",
      "Downloading: http://costco.com\n",
      "Downloading: http://secureserver.net\n",
      "Download error: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:833)\n",
      "Downloading: http://sputniknews.com\n",
      "Downloading: http://pornq.com\n",
      "Download error: Forbidden\n",
      "Downloading: http://libero.it\n",
      "Downloading: http://justdial.com\n",
      "Downloading: http://myfreecams.com\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-32:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 60, in download\n",
      "    response = opener.open(request)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 526, in open\n",
      "    response = self._open(req, data)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 544, in _open\n",
      "    '_open', req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 504, in _call_chain\n",
      "    result = func(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1346, in http_open\n",
      "    return self.do_open(http.client.HTTPConnection, req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1321, in do_open\n",
      "    r = h.getresponse()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 1331, in getresponse\n",
      "    response.begin()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 297, in begin\n",
      "    version, status, reason = self._read_status()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 258, in _read_status\n",
      "    line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\socket.py\", line 586, in readinto\n",
      "    return self._sock.recv_into(b)\n",
      "ConnectionResetError: [WinError 10054] 远程主机强迫关闭了一个现有的连接。\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://ticketmaster.com\n",
      "Downloading: http://europa.eu\n",
      "Download error: Forbidden\n",
      "Downloading: http://agoda.com\n",
      "Downloading: http://state.gov\n",
      "Downloading: http://telegraf.com.ua\n",
      "Downloading: http://yaplakal.com\n",
      "Downloading: http://126.com\n",
      "Downloading: http://impress.co.jp\n",
      "Downloading: http://uptobox.com\n",
      "Downloading: http://avast.com\n",
      "Downloading: http://thedailybeast.com\n",
      "Downloading: http://hellomagazine.com\n",
      "Downloading: http://jiameng.com\n",
      "Downloading: http://bicentenariobu.com\n",
      "Downloading: http://5ch.net\n",
      "Download error: [Errno 11001] getaddrinfo failed\n",
      "Downloading: http://utorrent.com\n",
      "Downloading: http://xe.com\n",
      "Download error: Forbidden\n",
      "Downloading: http://delta.com\n",
      "Download error: Forbidden\n",
      "Downloading: http://dafont.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://4399.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://hibids10.com\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-35:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 67, in download\n",
      "    return {'html':response.read().decode('utf-8'),'code':response.code}\n",
      "UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd3 in position 257: invalid continuation byte\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://redfin.com\n",
      "Download error: Forbidden\n",
      "Downloading: http://runoob.com\n",
      "Downloading: http://indiamart.com\n",
      "Downloading: http://chiphell.com\n",
      "Downloading: http://python.org\n",
      "Downloading: http://addthis.com\n",
      "Downloading: http://fmovies.se\n",
      "Downloading: http://kissasian.ch\n",
      "Downloading: http://epochtimes.com\n",
      "Downloading: http://intel.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://espncricinfo.com\n",
      "Download error: Service Temporarily Unavailable\n",
      "Downloading: http://kissasian.ch\n",
      "Downloading: http://hatenablog.com\n",
      "Download error: Service Temporarily Unavailable\n",
      "Downloading: http://beytoote.com\n",
      "Downloading: http://kizlarsoruyor.com\n",
      "Downloading: http://nextlnk2.com\n",
      "Downloading: http://cbsnews.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://lifewire.com\n",
      "Downloading: http://elmundo.es\n",
      "Downloading: http://dytt8.net\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-36:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 67, in download\n",
      "    return {'html':response.read().decode('utf-8'),'code':response.code}\n",
      "UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb5 in position 251: invalid start byte\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://gome.com.cn\n",
      "Downloading: http://bitbucket.org\n",
      "Downloading: http://mgid.com\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-39:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 67, in download\n",
      "    return {'html':response.read().decode('utf-8'),'code':response.code}\n",
      "UnicodeDecodeError: 'utf-8' codec can't decode byte 0xed in position 1142: invalid continuation byte\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://timeanddate.com\n",
      "Downloading: http://google.com.ly\n",
      "Downloading: http://animeyt.tv\n",
      "Downloading: http://gstatic.com\n",
      "Downloading: http://so3ody.com\n",
      "Downloading: http://nordstrom.com\n",
      "Download error: Forbidden\n",
      "Downloading: http://coursera.org\n",
      "Downloading: http://mathrubhumi.com\n",
      "Downloading: http://mercadolibre.com.ve\n",
      "Downloading: http://hubspot.com\n",
      "Downloading: http://wayfair.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://google.bg\n",
      "Downloading: http://people.com\n",
      "Download error: Forbidden\n",
      "Downloading: http://udn.com\n",
      "Downloading: http://naver.jp\n",
      "Downloading: http://viva.co.id\n",
      "Downloading: http://1bcde.com\n",
      "Download error: Forbidden\n",
      "Downloading: http://milliyet.com.tr\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-26:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 60, in download\n",
      "    response = opener.open(request)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 526, in open\n",
      "    response = self._open(req, data)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 544, in _open\n",
      "    '_open', req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 504, in _call_chain\n",
      "    result = func(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1346, in http_open\n",
      "    return self.do_open(http.client.HTTPConnection, req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1321, in do_open\n",
      "    r = h.getresponse()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 1331, in getresponse\n",
      "    response.begin()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 297, in begin\n",
      "    version, status, reason = self._read_status()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 258, in _read_status\n",
      "    line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\socket.py\", line 586, in readinto\n",
      "    return self._sock.recv_into(b)\n",
      "ConnectionResetError: [WinError 10054] 远程主机强迫关闭了一个现有的连接。\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://archiveofourown.org\n",
      "Downloading: http://hh.ru\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://discover.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://porn.com\n",
      "Downloading: http://corriere.it\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-34:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 67, in download\n",
      "    return {'html':response.read().decode('utf-8'),'code':response.code}\n",
      "UnicodeDecodeError: 'utf-8' codec can't decode byte 0xec in position 72353: invalid continuation byte\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://blog.me\n",
      "Downloading: http://6.cn\n",
      "Downloading: http://lemonde.fr\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-43:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 60, in download\n",
      "    response = opener.open(request)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 526, in open\n",
      "    response = self._open(req, data)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 544, in _open\n",
      "    '_open', req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 504, in _call_chain\n",
      "    result = func(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1346, in http_open\n",
      "    return self.do_open(http.client.HTTPConnection, req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1321, in do_open\n",
      "    r = h.getresponse()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 1331, in getresponse\n",
      "    response.begin()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 297, in begin\n",
      "    version, status, reason = self._read_status()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 258, in _read_status\n",
      "    line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\socket.py\", line 586, in readinto\n",
      "    return self._sock.recv_into(b)\n",
      "ConnectionResetError: [WinError 10054] 远程主机强迫关闭了一个现有的连接。\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://hespress.com\n",
      "Downloading: http://lifehacker.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://uniqlo.tmall.com\n",
      "Downloading: http://feedly.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://superuser.com\n",
      "Downloading: http://cryptobrowser.site\n",
      "Download error: The HTTP server returned a redirect error that would lead to an infinite loop.\n",
      "The last 30x error message was:\n",
      "\n",
      "Downloading: http://discuss.com.hk\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-30:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 60, in download\n",
      "    response = opener.open(request)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 526, in open\n",
      "    response = self._open(req, data)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 544, in _open\n",
      "    '_open', req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 504, in _call_chain\n",
      "    result = func(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1346, in http_open\n",
      "    return self.do_open(http.client.HTTPConnection, req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1321, in do_open\n",
      "    r = h.getresponse()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 1331, in getresponse\n",
      "    response.begin()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 297, in begin\n",
      "    version, status, reason = self._read_status()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 258, in _read_status\n",
      "    line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\socket.py\", line 586, in readinto\n",
      "    return self._sock.recv_into(b)\n",
      "ConnectionResetError: [WinError 10054] 远程主机强迫关闭了一个现有的连接。\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://vnexpress.net\n",
      "Downloading: http://kayak.com\n",
      "Downloading: http://unsplash.com\n",
      "Downloading: http://rbc.ru\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://investopedia.com\n",
      "Downloading: http://moneycontrol.com\n",
      "Downloading: http://bandcamp.com\n",
      "Downloading: http://douyin.com\n",
      "Downloading: http://media.tumblr.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://eksisozluk.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://nhentai.net\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-41:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 60, in download\n",
      "    response = opener.open(request)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 526, in open\n",
      "    response = self._open(req, data)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 544, in _open\n",
      "    '_open', req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 504, in _call_chain\n",
      "    result = func(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1346, in http_open\n",
      "    return self.do_open(http.client.HTTPConnection, req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1321, in do_open\n",
      "    r = h.getresponse()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 1331, in getresponse\n",
      "    response.begin()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 297, in begin\n",
      "    version, status, reason = self._read_status()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 258, in _read_status\n",
      "    line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\socket.py\", line 586, in readinto\n",
      "    return self._sock.recv_into(b)\n",
      "ConnectionResetError: [WinError 10054] 远程主机强迫关闭了一个现有的连接。\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://merriam-webster.com\n",
      "Downloading: http://torrentz2.eu\n",
      "Download error: Service Temporarily Unavailable\n",
      "Downloading: http://torrentz2.eu\n",
      "Download error: Service Temporarily Unavailable\n",
      "Downloading: http://alnaharegypt.com\n",
      "Downloading: http://advbanners.global.ssl.fastly.net\n",
      "Downloading: http://eyny.com\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-45:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 60, in download\n",
      "    response = opener.open(request)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 526, in open\n",
      "    response = self._open(req, data)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 544, in _open\n",
      "    '_open', req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 504, in _call_chain\n",
      "    result = func(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1346, in http_open\n",
      "    return self.do_open(http.client.HTTPConnection, req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1321, in do_open\n",
      "    r = h.getresponse()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 1331, in getresponse\n",
      "    response.begin()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 297, in begin\n",
      "    version, status, reason = self._read_status()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 258, in _read_status\n",
      "    line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\socket.py\", line 586, in readinto\n",
      "    return self._sock.recv_into(b)\n",
      "ConnectionResetError: [WinError 10054] 远程主机强迫关闭了一个现有的连接。\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://npr.org\n",
      "Downloading: http://ilovepdf.com\n",
      "Downloading: http://google.rs\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://t-online.de\n",
      "Download error: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:833)\n",
      "Downloading: http://binance.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://vkuseraudio.net\n",
      "Download error: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:833)\n",
      "Downloading: http://google.tm\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://kp.ru\n",
      "Downloading: http://tamilrockers.cl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-47:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 67, in download\n",
      "    return {'html':response.read().decode('utf-8'),'code':response.code}\n",
      "UnicodeDecodeError: 'utf-8' codec can't decode bytes in position 94503-94505: invalid continuation byte\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://gyazo.com\n",
      "Downloading: http://cnnindonesia.com\n",
      "Downloading: http://pandora.com\n",
      "Downloading: http://animeflv.net\n",
      "Downloading: http://sabq.org\n",
      "Downloading: http://zukxd6fkxqn.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://3dmgame.com\n",
      "Downloading: http://zougla.gr\n",
      "Downloading: http://asana.com\n",
      "Downloading: http://olx.com.br\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-48:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 67, in download\n",
      "    return {'html':response.read().decode('utf-8'),'code':response.code}\n",
      "UnicodeDecodeError: 'utf-8' codec can't decode byte 0xfa in position 242: invalid start byte\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://torrent9.blue\n",
      "Download error: Service Temporarily Unavailable\n",
      "Downloading: http://torrent9.blue\n",
      "Download error: Service Temporarily Unavailable\n",
      "Downloading: http://interia.pl\n",
      "Downloading: http://techradar.com\n",
      "Downloading: http://namasha.com\n",
      "Downloading: http://norton.com\n",
      "Downloading: http://artstation.com\n",
      "Downloading: http://fifa.com\n",
      "Downloading: http://telegraph.co.uk\n",
      "Downloading: http://wtoip.com\n",
      "Downloading: http://jw.org\n",
      "Downloading: http://southwest.com\n",
      "Download error: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:833)\n",
      "Downloading: http://hotnewhiphop.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://brilio.net\n",
      "Downloading: http://dawn.com\n",
      "Downloading: http://www.gov.uk\n",
      "Downloading: http://blatungo.com\n",
      "Downloading: http://banesconline.com\n",
      "Download error: [Errno 11001] getaddrinfo failed\n",
      "Downloading: http://telewebion.com\n",
      "Downloading: http://groupon.com\n",
      "Downloading: http://thefreedictionary.com\n",
      "Downloading: http://theverge.com\n",
      "Downloading: http://webmd.com\n",
      "Downloading: http://newegg.com\n",
      "Downloading: http://hupu.com\n",
      "Downloading: http://egy.best\n",
      "Downloading: http://pexels.com\n",
      "Downloading: http://mit.edu\n",
      "Downloading: http://crunchyroll.com\n",
      "Downloading: http://smallpdf.com\n",
      "Downloading: http://bild.de\n",
      "Downloading: http://ptt.cc\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://tube8.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://wunderground.com\n",
      "Downloading: http://drive2.ru\n",
      "Downloading: http://bancodevenezuela.com\n",
      "Downloading: http://kijiji.ca\n",
      "Downloading: http://google.com.do\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://as.com\n",
      "Downloading: http://goal.com\n",
      "Downloading: http://nypost.com\n",
      "Downloading: http://grammarly.com\n",
      "Downloading: http://cricbuzz.com\n",
      "Downloading: http://blog.jp\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-49:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 60, in download\n",
      "    response = opener.open(request)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 526, in open\n",
      "    response = self._open(req, data)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 544, in _open\n",
      "    '_open', req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 504, in _call_chain\n",
      "    result = func(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1346, in http_open\n",
      "    return self.do_open(http.client.HTTPConnection, req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1321, in do_open\n",
      "    r = h.getresponse()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 1331, in getresponse\n",
      "    response.begin()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 297, in begin\n",
      "    version, status, reason = self._read_status()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 258, in _read_status\n",
      "    line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\socket.py\", line 586, in readinto\n",
      "    return self._sock.recv_into(b)\n",
      "ConnectionResetError: [WinError 10054] 远程主机强迫关闭了一个现有的连接。\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://scribol.com\n",
      "Downloading: http://prothomalo.com\n",
      "Downloading: http://dianping.com\n",
      "Downloading: http://autodesk.com\n",
      "Downloading: http://sci-hub.tw\n",
      "Downloading: http://qingdaonews.com\n",
      "Downloading: http://velocecdn.com\n",
      "Download error: Forbidden\n",
      "Downloading: http://quizlet.com\n",
      "Downloading: http://geeksforgeeks.org\n",
      "Downloading: http://cqnews.net\n",
      "Downloading: http://mercari.com\n",
      "Downloading: http://eventbrite.com\n",
      "Downloading: http://paytm.com\n",
      "Downloading: http://att.com\n",
      "Downloading: http://namnak.com\n",
      "Downloading: http://alicdn.com\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-50:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 60, in download\n",
      "    response = opener.open(request)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 526, in open\n",
      "    response = self._open(req, data)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 544, in _open\n",
      "    '_open', req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 504, in _call_chain\n",
      "    result = func(*args)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1346, in http_open\n",
      "    return self.do_open(http.client.HTTPConnection, req)\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\urllib\\request.py\", line 1321, in do_open\n",
      "    r = h.getresponse()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 1331, in getresponse\n",
      "    response.begin()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 297, in begin\n",
      "    version, status, reason = self._read_status()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\http\\client.py\", line 266, in _read_status\n",
      "    raise RemoteDisconnected(\"Remote end closed connection without\"\n",
      "http.client.RemoteDisconnected: Remote end closed connection without response\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://ieee.org\n",
      "Downloading: http://youth.cn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-51:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 67, in download\n",
      "    return {'html':response.read().decode('utf-8'),'code':response.code}\n",
      "UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd6 in position 98: invalid continuation byte\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://cisco.com\n",
      "Download error: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:833)\n",
      "Downloading: http://appledaily.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://cinecalidad.to\n",
      "Downloading: http://express.co.uk\n",
      "Downloading: http://wattpad.com\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://ultimate-guitar.com\n",
      "Downloading: http://convert2mp3.net\n",
      "Downloading: http://xda-developers.com\n",
      "Downloading: http://google.lk\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://hotels.com\n",
      "Downloading: http://kooora.com\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-52:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Program Files\\Python36\\Lib\\threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-4-fd4ed9c52be6>\", line 25, in process_queue\n",
      "    html = D(url)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 45, in __call__\n",
      "    result = self.download(url, headers, proxy, self.num_retries)\n",
      "  File \"<ipython-input-2-a8f67498e3b5>\", line 67, in download\n",
      "    return {'html':response.read().decode('utf-8'),'code':response.code}\n",
      "UnicodeDecodeError: 'utf-8' codec can't decode byte 0xdf in position 78: invalid continuation byte\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://gosuslugi.ru\n",
      "Downloading: http://emol.com\n",
      "Downloading: http://google.by\n",
      "Download error: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。\n",
      "Downloading: http://gamersky.com\n",
      "Downloading: http://idntimes.com\n",
      "Downloading: http://pantip.com\n",
      "Downloading: http://healthline.com\n",
      "Downloading: http://engadget.com\n",
      "Downloading: http://bleacherreport.com\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-6-11ba697769ca>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mstart\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m# record start time of scrape\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mthreaded_crawler\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murls\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'%.2f seconds'\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mend\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0mstart\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-4-fd4ed9c52be6>\u001b[0m in \u001b[0;36mthreaded_crawler\u001b[1;34m(send_url, delay, cache, max_threads)\u001b[0m\n\u001b[0;32m     33\u001b[0m         \u001b[1;31m# all threads have been processed\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     34\u001b[0m         \u001b[1;31m# sleep temporarily so CPU can focus execution elsewhere\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 35\u001b[1;33m         \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mSLEEP_TIME\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "start = time.time() # record start time of scrape\n",
    "threaded_crawler(urls)\n",
    "end = time.time()\n",
    "print('%.2f seconds' % (end-start)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime, timedelta\n",
    "from pymongo import MongoClient, errors\n",
    "\n",
    "class MongoQueue:\n",
    "    # possible states of a download\n",
    "    OUTSTANDING, PROCESSING, COMPLETE = range(3)\n",
    "    def __init__(self, client=None, timeout=300):\n",
    "        self.client = MongoClient() if client is None else client\n",
    "        self.db = self.client.cache\n",
    "        self.timeout = timeout\n",
    "    def __nonzero__(self):\n",
    "        '''\n",
    "        Returns True if there are more jobs to process\n",
    "        '''\n",
    "        record = self.db.crawl_queue.find_one(\n",
    "            {'status': {'$ne': self.COMPLETE}}\n",
    "        )\n",
    "        return True if record else False\n",
    "    def push(self, url):\n",
    "        '''\n",
    "        Add new URL to queue if does not exist\n",
    "        '''\n",
    "        try:\n",
    "            self.db.crawl_queue.insert({'_id': url, 'status':self.OUTSTANDING})\n",
    "        except errors.DuplicateKeyError as e:\n",
    "            pass # this is already in the queue\n",
    "    def pop(self):\n",
    "        '''\n",
    "        Get an outstanding URL from the queue and set its status to processing. If the queue is empty a KeyError exception is raised.\n",
    "        '''\n",
    "        record = self.db.crawl_queue.find_and_modify(\n",
    "            query={'status': self.OUTSTANDING},\n",
    "            update={'$set': {'status': self.PROCESSING, 'timestamp': datatime.now()}}\n",
    "        )\n",
    "        if record:\n",
    "            return record['_id']\n",
    "        else:\n",
    "            self.repair()\n",
    "            raise KeyError()\n",
    "    def complete(self, url):\n",
    "        self.db.crawl_queue.update({'_id': url}, {'$set':{'status': self.COMPLETE}})\n",
    "    def repair(self):\n",
    "        '''\n",
    "        Release stalled jobs\n",
    "        '''\n",
    "        record = self.db.crawl_queue.find_and_modify(\n",
    "            query={\n",
    "                'timestamp': {'$lt': datetime.now()-timedelta(seconds=self.timeout)},\n",
    "                'status': {'$ne': self.COMPLETE}\n",
    "            },\n",
    "            update={'$set': {'status': self.OUTSTANDING}}\n",
    "        )\n",
    "        if record:\n",
    "            print('Released:', record['_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import multiprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_cpus = multiprocessing.cpu_count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_cpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
